package util.ranker;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.lucene.queryParser.ParseException;

import util.evaluation.QueryTag;
import util.evaluation.StatUnit;
import util.evaluation.StatsGenerator;
import util.hashing.Sorting;
import util.io.FileInput;
import util.io.FileOutput;
import util.ir.LMSearcherMix;
import util.models.Tags;

public class LocalLauncherMix {

	/**
	 * @param args
	 */
	
	
	String prefix_index_raw="/Users/sergioduarte/projects/delicious_index_raw/";
	
	String prefix_index="/Users/sergioduarte/projects/delicious_index/";
	
	String prefix_data="/Users/sergioduarte/projects/data/";

	//String prefix_data="/home/sergio/data/";
	//String prefix_index_raw="/home/sergio/projects/delicious_index/";
	
	//String prefix_index="/home/sergio/delicious_index/";
	
	
	
	public String index_raw = prefix_index_raw + "bigrams_mix_clean2/bigrams_mix_clean2.txt";
	String bigram_index = prefix_index + "bigrams_index_mix_clean2";
	String trigram_index = prefix_index + "trigrams_index_mix";

	String bigram_raw = prefix_index_raw+ "bigrams_mix/bigrams_enriched.txt";

	// a american air www.aa.com/aa/reservation/roundTripSearchAccess.do
	// years_10_12 kids

	int _eval_query = 0;
	int _eval_suggestion = 1;
	int _eval_url = 2;
	int _eval_age = 3;
	int _eval_bucket = 4;

	//
	int _q = 0;
	int _tag = 1;
	int _freq = 2;
	int _freq_collection = 4;
	int _rank = 4;

	// 1 performances 1 1595 107

	String out_path = prefix_data+ "ranked_MixModel_kids/ranking_kids.txt";
	FileOutput output = new FileOutput(out_path);

	String evaluation_path = prefix_data +   "results/queryTestSet_clicks_kids_sample2/queryTestSet_clicks_kids_sample2.txt";
	String in_path = prefix_data+ "results/query_and_tags_sorted_clicks_kids/query_and_tags_sorted_clicks_kids.txt";

	Hashtable<String, ArrayList<QueryTag>> queries = new Hashtable<String, ArrayList<QueryTag>>();
	HashSet<String> queries_to_eval = new HashSet<String>();

	HashSet<String> vocabulary = new HashSet<String>();
	static Tags tags = new Tags();
	String target = "teens-mteens";
	String target_b = "kids-teens-mteens";
	LMSearcherMix trigram_searcher;
	LMSearcherMix bigram_searcher;

	private int query_threshold = 200;
	private int query_freq_threshold = 1;

	BufferedWriter out = null;

	public LocalLauncherMix() {

		try {
			out = new BufferedWriter(new FileWriter(out_path));
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		try {
			initChildrenQueries();
			initQueries();
			// initVocabularyMix();

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		initTagsFrequency(index_raw);
	}

	public void initVocabularyMix1() {

		FileInput in = new FileInput(bigram_raw);

		String line = in.readString();

		while (line != null) {

			String t[] = line.split("\t");

			String a = t[1];
			String b = t[2];

			if (t[0].equals("kids-teens-mteens")) {
				vocabulary.add(a);
				vocabulary.add(b);
				// System.out.println("adding" + "\t" + a + "\t"+ b);
			}

			line = in.readString();

		}

	}

	public void initChildrenQueries() {

		FileInput in = new FileInput(evaluation_path);

		String line = in.readString();

		while (line != null) {

			String t[] = line.split("\t");
			String query = t[_eval_query];
			String bucket = t[_eval_bucket];
			if (bucket.equals("kids")) {
				queries_to_eval.add(query);

			}

			line = in.readString();

		}

	}

	public void initQueries() throws IOException {

		// trigram_searcher = new LMSearcherMix(trigram_index);
		bigram_searcher = new LMSearcherMix(bigram_index);

		FileInput in = new FileInput(in_path);
		String line = in.readString();
		while (line != null) {
			String t[] = line.split("\t");

			QueryTag qt = new QueryTag();

			qt.tag = t[_tag];
			qt.freq = Integer.valueOf(t[_freq]);
			// qt.collection_freq = Integer.valueOf(t[_freq_collection]);
			// qt.rank = Integer.valueOf(t[_rank]);

			String query = t[_q];

			// only add tag information of queries that are going to be
			// evaluatedresults.tar
			if (!queries_to_eval.contains(query)) {
				line = in.readString();
				continue;
			}

			if (queries.containsKey(query)) {

				queries.get(query).add(qt);
			} else {

				ArrayList<QueryTag> temp = new ArrayList<QueryTag>();
				temp.add(qt);
				queries.put(query, temp);
			}

			line = in.readString();

		}

	}

	public Hashtable<String, ArrayList<String>> rankQueries() {

		Enumeration<String> keys = queries.keys();
		int query_index = 1;
		Hashtable<String, ArrayList<String>> suggs = new Hashtable<String, ArrayList<String>>();
		while (keys.hasMoreElements()) {

			String query = keys.nextElement();

			if (!queries_to_eval.contains(query)) {

				continue;
			}

			try {

				// if(!query.equals("star wars")) continue;

				Iterator<Entry<String, Float>> ranked = rankQuery(query);
				int rank = 1;

				// printing ranked suggestions for query
				while (ranked != null && ranked.hasNext()) {

					Entry<String, Float> suggestion = ranked.next();
					String suggested = suggestion.getKey();
					suggested= suggested.replace("_", " ");
					
					updateSuggestionHash(suggs, query, suggested);
					
					
					String temp = query + "\t" + suggested + "\t"
							+ rank + "\n";

					out.write(temp);
					// System.out.println(query + "\t" + suggestion.getKey()+
					// "\t" + rank);
					rank++;
				}
				System.out.println(query_index + "\t" + "Query:" + query
						+ "\tFound:" + rank);
				query_index++;

			} catch (IOException e) {
				// TODO Auto-generated catch block

				e.printStackTrace();
				System.exit(1);
			} catch (ParseException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				System.exit(1);
			}

		}

		try {
			out.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
		return suggs;
	}

	private void updateSuggestionHash(
			Hashtable<String, ArrayList<String>> suggs,
			String query, String suggested) {
		// TODO Auto-generated method stub
		
		 ArrayList<String>temp = new ArrayList<String>();
		
		if(suggs.containsKey(query)){
			
			temp = suggs.get(query);
			
			
		}
	suggs.put(query, temp);
	  temp.add(suggested);
	}

	private Iterator<Entry<String, Float>> rankQuery(String query)
			throws IOException, ParseException {
		// TODO Auto-generated method stub

		Hashtable<String, Float> q = new Hashtable<String, Float>();

		q = buildQuery(query, false,10);

		TagRankingMix ranker = new TagRankingMix(trigram_searcher,
				bigram_searcher, target, "adults", target_b, q, query);

		Iterator<Entry<String, Float>> ranked = ranker.rankSuggestions();

		// what is below is temporal
		// Enumeration<String> keys = q.keys();
		// Hashtable<String, Float> qq = new Hashtable<String, Float>();
		// while (keys.hasMoreElements()) {//

		// String tag = keys.nextElement();
		// if (vocabulary.contains(tag)) {

		// qq.put(tag, q.get(tag));
		// }

		// }
		// Iterator<Entry<String, Float>> ranked =
		// util.hashing.Sorting.sortHashNumericValues(qq, true);

		// Iterator<Entry<String, Float>> ranked= null;
		return ranked;
	}

	private Iterator<Entry<String, Float>> rankQuerySimple(String query)
			throws IOException, ParseException {
		// TODO Auto-generated method stub

		Hashtable<String, Float> q = new Hashtable<String, Float>();

		q = buildQuery(query, true,100);

		return Sorting.sortHashNumericValues(q, false);
		// Iterator<Entry<String, Float>> ranked = ranker.rankSuggestions();

	}

	public Hashtable<String, Float> subQuery(Hashtable<String, Float> query,
			String query_text) {

		Hashtable<String, Float> qq = new Hashtable<String, Float>();

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

		}
		return qq;

	}

	public static void addSingleTermsFreq(Hashtable<String, Float> query) {

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			String tokens[] = tag.split("_");

			for (int i = 0; i < tokens.length; i++) {

				if (query.containsKey(tokens[i])) {

					query.put(tokens[i], query.get(tokens[i]) + 1);
				}

			}

		}

	}

	public static Hashtable<String, Float> normalize(
			Hashtable<String, Float> query) {

		Hashtable<String, Float> q = new Hashtable<String, Float>();
		Float m[] = findMinAndMax(query);

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

			Float value = query.get(key);

			value = (value - m[0]) / (m[1] - m[0]);

			q.put(key, value);

		}

		return q;

	}

	public static Float[] findMinAndMax(Hashtable<String, Float> query) {
		Float[] m = new Float[2];

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String key = keys.nextElement();

			if (m[0] == null) {

				m[0] = m[1] = query.get(key);

			} else {

				float value = query.get(key);

				if (value < m[0]) {

					m[0] = value;
				}

				if (value > m[1]) {

					m[1] = value;
				}
			}

		}

		return m;

	}

	public static Hashtable<String, Float> calculateKL(
			Hashtable<String, Float> query) {

		Hashtable<String, Float> kl = new Hashtable<String, Float>();

		Enumeration<String> keys = query.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			if (tag.length() <= 2) {

				continue;
			}

			int freq_s_foreground = tags.getFreq("kids", tag);

			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);

			int freq_s_background = tags.getFreq("adults", tag);

			freq_s_background = freq_s_background + tags.getFreq("back", tag);
			freq_s_background = freq_s_background + freq_s_foreground;

			float prob_s_fore = freq_s_foreground / (float) TOTALS.KIDS_UNIGRAM;
			float prob_s_back = freq_s_background
					/ (float) TOTALS.ADULTS_UNIGRAM;

			if (prob_s_fore == 0.0f) {

				continue;
			}

			float pmi = -1f;
			if (prob_s_fore != 0.0f) {
				pmi = (float) Math.log(prob_s_fore / prob_s_back);
				pmi = prob_s_fore * pmi;

				// float sign= Math.signum(pmi);
				// pmi = (float) Math.log(Math.abs(pmi));
				// pmi = pmi *sign;
				// pmi = prob_s_fore;

			}

			kl.put(tag, pmi);
			// System.out.println("pmi\t"+tag+"\t"+ pmi);

		}

		// return kl;
		return normalize(kl);

	}

	public static Hashtable<String, Float> calculateQueryProb(
			Hashtable<String, Float> query_freq, float total_freq) {

		Hashtable<String, Float> kl = new Hashtable<String, Float>();

		Enumeration<String> keys = query_freq.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();

			if (tag.length() <= 2) {

				continue;
			}

			Float freq = query_freq.get(tag);

			int freq_s_foreground = tags.getFreq("kids", tag);

			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);

			int freq_s_background = tags.getFreq("adults", tag);

			freq_s_background = freq_s_background + tags.getFreq("back", tag);
			freq_s_background = freq_s_background + freq_s_foreground;

			float prob_s_fore = freq_s_foreground / (float) TOTALS.KIDS_UNIGRAM;
			float prob_s_back = freq_s_background
					/ (float) TOTALS.ADULTS_UNIGRAM;

			float prob_q = (float) freq / total_freq;
			// prob_q = (float)Math.log(prob_q);
			// pmi = pmi+prob_q;

			float w = (float) Math.log(1 / prob_s_fore);
			float score = prob_q * w;
			// score = prob_q;

			if (prob_s_fore == 0.0f) {

				continue;
			}

			kl.put(tag, score);

			// System.out.println("prob_q " + tag + "\t" + score);
		}
		// return kl;
		return normalize(kl);

	}

	public Hashtable<String, Float> buildQuery(String query_text, boolean scores, int limit) {

		Hashtable<String, Float> query = new Hashtable<String, Float>();
		Hashtable<String, Float> query_freq = new Hashtable<String, Float>();
		float total_freq = 0.0f;
		ArrayList<QueryTag> list = queries.get(query_text);

		for (int i = 0; i < list.size() && i <= query_threshold
				&& list.get(i).freq >= query_freq_threshold; i++) {
			String tag = list.get(i).tag;

			// customize this part of code

			float freq_s_foreground = tags.getFreq("kids", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("kids-teens-mteens", tag);
			freq_s_foreground = freq_s_foreground
					+ tags.getFreq("teens-mteens", tag);

			// end of customizing part to be DONE

			if (freq_s_foreground == 0) {

				continue;
			}

			if (tag.length() <= 2) {

				continue;
			}

			query_freq.put(list.get(i).tag, (float) list.get(i).freq);

			total_freq += (float) list.get(i).freq;

		}

		// getting query tag and frequencies in the query

		Hashtable<String, Float> kl = calculateKL(query_freq);
		Hashtable<String, Float> query_prob = calculateQueryProb(query_freq,
				total_freq);

		Float THRESHOLD = 1f;

		Enumeration<String> keys = kl.keys();

		while (keys.hasMoreElements()) {

			String tag = keys.nextElement();
			float kl_value = kl.get(tag);

			float qp_value = query_prob.get(tag);
			if (tag.length() <= 2) {

				continue;
			}

			float freq = query_freq.get(tag);
			if (freq >= THRESHOLD) {

				float score = qp_value;

				score = score * (kl_value);
				score = score;
				// System.out.println(tag+"\t"+prob_s_fore + "\t"+ prob_s_back +
				// "\t"+ pmi + "\t" +prob_q + "\t"+ score);
				query.put(tag, score);
				// System.out.println(t[1]);
				// query.put(t[1], Float.valueOf(t[2]));

			}

		}

		Iterator<Entry<String, Float>> sorted = Sorting.sortHashNumericValues(
				query, false);

		Hashtable<String, Float> q_sorted = new Hashtable<String, Float>();

		int threshold = 1;

		int j = 0;
		while (sorted.hasNext() && j < limit) {
			j++;
			Entry<String, Float> n = sorted.next();
			float value = query_freq.get(n.getKey());
			if (scores) {

				value = n.getValue();
			}
			// System.out.println("***Query representation\t" + query_text +
			// "\t" +n.getKey() + "\t" + query_freq.get(n.getKey()));
			q_sorted.put(n.getKey(), value);

		}

		return q_sorted;
	}

	public static void initTagsFrequency(String path) {
		// adults 0 1 123 19138 14231 292 5186 34153

		FileInput in = new FileInput(path);
		String line = in.readString();

		while (line != null) {

			String t[] = line.split("\t");

			tags.addFreq(t[0], t[1], Integer.valueOf(t[4]));
			tags.addFreq(t[0], t[2], Integer.valueOf(t[5]));

			tags.addFreq("back", t[1], Integer.valueOf(t[7]));
			tags.addFreq("back", t[2], Integer.valueOf(t[8]));

			line = in.readString();

		}

	}

	private Hashtable<String, Float> buildQuery1(String query) {
		// TODO Auto-generated method stub

		Hashtable<String, Float> q = new Hashtable<String, Float>();

		ArrayList<QueryTag> list = queries.get(query);

		for (int i = 0; i < list.size() && i <= query_threshold
				&& list.get(i).freq >= query_freq_threshold; i++) {

			q.put(list.get(i).tag, (float) list.get(i).freq);

		}

		if (q.size() == 0) {

			for (int i = 0; i < list.size() && i <= query_threshold; i++) {

				q.put(list.get(i).tag, (float) list.get(i).freq);

			}

		}

		return q;

	}

	// Output query suggestion rank

	
	
	public void printStats(Hashtable<String, StatUnit> stats){
		
		
		Enumeration<String> keys = stats.keys();
		
		while(keys.hasMoreElements()){
			String type = keys.nextElement();
			StatUnit unit  = stats.get(type);
			System.out.println(type + "\t"+ unit.matches + "\t"+ unit.total_queries + "\t" + unit.total + "\t" +unit.precision()+ "\t"+ unit.precision_relaxed()+ "\t"+ unit.rankAverage());
			
		}
		
	}
	
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub

		LocalLauncherMix launcher = new LocalLauncherMix();

		
	}

}

